| Disk ARchive 2.8.1 Full featured and portable backup and archiving tool

Loading...
Searching...
No Matches
parallel_tronconneuse.hpp
Go to the documentation of this file.
1/*********************************************************************/
2// dar - disk archive - a backup/restoration program
3// Copyright (C) 2002-2025 Denis Corbin
4//
5// This program is free software; you can redistribute it and/or
6// modify it under the terms of the GNU General Public License
7// as published by the Free Software Foundation; either version 2
8// of the License, or (at your option) any later version.
9//
10// This program is distributed in the hope that it will be useful,
11// but WITHOUT ANY WARRANTY; without even the implied warranty of
12// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13// GNU General Public License for more details.
14//
15// You should have received a copy of the GNU General Public License
16// along with this program; if not, write to the Free Software
17// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
18//
19// to contact the author, see the AUTHOR file
20/*********************************************************************/
21
25
34
35#ifndef PARALLEL_TRONCONNEUSE_HPP
36#define PARALLEL_TRONCONNEUSE_HPP
37
38#include "../my_config.h"
39#include <string>
40
41#include "infinint.hpp"
42#include "archive_version.hpp"
43#include "crypto_segment.hpp"
44#include "heap.hpp"
45#include "crypto_module.hpp"
46#include "proto_tronco.hpp"
47
48#include <libthreadar/libthreadar.hpp>
49
50 namespace libdar
51{
52
55
56 // those class are used by the parallel_tronconneuse class to wrap the different
57 // type of threads. They are defined just after the parallel_tronconneuse definition
58 class read_below;
59 class write_below;
60 class crypto_worker;
61
63
64 enum class tronco_flags { normal = 0, stop = 1, eof = 2, die = 3, data_error = 4, exception_below = 5, exception_worker = 6, exception_error = 7 };
65
66
68 //
69 // the parallel_tronconneuse class that orchestrate all that
70 //
71 //
72
73
75
87
88 class parallel_tronconneuse : public proto_tronco
89 {
90 public:
92
100 parallel_tronconneuse(U_I workers,
101 U_32 block_size,
102 generic_file & encrypted_side,
103 const archive_version & reading_ver,
104 std::unique_ptr<crypto_module> & ptr);
105
107 parallel_tronconneuse(const parallel_tronconneuse & ref) = delete;
108
110 parallel_tronconneuse(parallel_tronconneuse && ref) = default;
111
113 parallel_tronconneuse & operator = (const parallel_tronconneuse & ref) = delete;
114
116 parallel_tronconneuse & operator = (parallel_tronconneuse && ref) noexcept = default;
117
119 ~parallel_tronconneuse() noexcept;
120
122 virtual bool skippable(skippability direction, const infinint & amount) override;
124 virtual bool skip(const infinint & pos) override;
126 virtual bool skip_to_eof() override;
128 virtual bool skip_relative(S_I x) override;
130 virtual bool truncatable(const infinint & pos) const override { return false; };
132 virtual infinint get_position() const override { if(is_terminated()) throw SRC_BUG; return current_position; };
133
135
140 virtual void write_end_of_file() override { if(is_terminated()) throw SRC_BUG; sync_write(); };
141
142
144
145 virtual void set_initial_shift(const infinint & x) override;
146
150 virtual void set_callback_trailing_clear_data(trailing_clear_data_callback call_back) override;
151
153 virtual U_32 get_clear_block_size() const override { return clear_block_size; };
154
155 private:
156
157 // inherited from generic_file
158
160 virtual void inherited_read_ahead(const infinint & amount) override;
161
163 virtual U_I inherited_read(char *a, U_I size) override;
164
166
168 virtual void inherited_write(const char *a, U_I size) override;
169
171
174 virtual void inherited_truncate(const infinint & pos) override { throw SRC_BUG; };
175
177 virtual void inherited_sync_write() override;
178
179
181 virtual void inherited_flush_read() override;
182
184 virtual void inherited_terminate() override;
185
186 const archive_version & get_reading_version() const { return reading_ver; };
187
188 // internal data structure
189 enum class thread_status { running, suspended, dead };
190
191 // the fields
192
193 U_I num_workers;
194 U_32 clear_block_size;
195 infinint current_position;
196 infinint initial_shift;
197 archive_version reading_ver;
198 std::unique_ptr<crypto_module> crypto;
199 infinint (*mycallback)(generic_file & below, const archive_version & reading_ver);
200 generic_file* encrypted;
201
202 // fields used to represent possible status of subthreads and communication channel (the pipe)
203
204 U_I ignore_stop_acks;
205 thread_status t_status;
206
207
208 // the following stores data from the ratelier_gather to be provided for read() operation
209 // the lus_data/lus_flags is what is extracted from the ratelier_gather, both constitute
210 // the feedback channel from sub-threads to provide order acks and normal data
211
212 std::deque<std::unique_ptr<crypto_segment> > lus_data;
213 std::deque<signed int> lus_flags;
214 bool lus_eof;
215 bool check_bytes_to_skip;
216
217 // the following stores data going to ratelier_scatter for the write() operation
218
219 std::unique_ptr<crypto_segment> tempo_write;
220 infinint block_num;
221
222 // the datastructures shared among threads
223
224 std::shared_ptr<libthreadar::ratelier_scatter<crypto_segment> > scatter;
225 std::shared_ptr<libthreadar::ratelier_gather<crypto_segment> > gather;
226 std::shared_ptr<libthreadar::barrier> waiter;
227 std::shared_ptr<heap<crypto_segment> > tas;
228
229 // the child threads
230
231 std::deque<std::unique_ptr<crypto_worker> > travailleur;
232 std::unique_ptr<read_below> crypto_reader;
233 std::unique_ptr<write_below> crypto_writer;
234
235
236
238
249 bool send_read_order(tronco_flags order, const infinint & for_offset = 0);
250
252 void send_write_order(tronco_flags order);
253
255 void go_read();
256
258 void read_refill();
259
261
267 tronco_flags purge_ratelier_from_next_order(infinint pos = 0);
268
270
282 bool purge_unack_stop_order(const infinint & pos = 0);
283
285
293 bool find_offset_in_lus_data(const infinint & pos);
294
296 void run_threads();
297
299 void stop_threads();
300
302 void join_workers_only();
303
305 void join_threads();
306
307
308 static U_I get_ratelier_size(U_I num_worker) { return num_worker + num_worker/2; };
309 static U_I get_heap_size(U_I num_worker);
310 };
311
312
314 //
315 // read_below subthread used by parallel_tronconneuse
316 // to dispatch chunk of encrypted data to the workers
317 //
318
319 class read_below: public libthreadar::thread
320 {
321 public:
322 read_below(const std::shared_ptr<libthreadar::ratelier_scatter<crypto_segment> > & to_workers,
323 const std::shared_ptr<libthreadar::barrier> & waiter,
324 U_I num_workers,
325 U_I clear_block_size,
326 generic_file* encrypted_side,
327 const std::shared_ptr<heap<crypto_segment> > xtas,
328 infinint init_shift
329 );
330
331 ~read_below() { if(ptr) tas->put(std::move(ptr)); cancel(); join(); };
332
336 void set_callback_trailing_clear_data(trailing_clear_data_callback call_back) { trailing_clear_data = call_back; };
337
338 // *** //
339 // *** the method above should not be used anymore once the thread is running *** //
340 // *** //
341
343 void set_initial_shift(const infinint & x) { initial_shift = x; };
344
346
351 void set_pos(const infinint & pos) { skip_to = pos; };
352
354
363 void set_flag(tronco_flags val) { flag = val; };
364
366
371 const infinint & get_clear_flow_start() const { return clear_flow_start; };
372
374
380 const infinint & get_pos_in_flow() const { return pos_in_flow; };
381
383 void read_ahead_up_to(const infinint & offset);
384
385
386 protected:
387 virtual void inherited_run() override;
388
389 private:
390 std::shared_ptr<libthreadar::ratelier_scatter <crypto_segment> > workers;
391 std::shared_ptr<libthreadar::barrier> waiting;
392 U_I num_w;
393 U_I clear_buf_size;
394 generic_file* encrypted;
395 archive_version version;
396 std::shared_ptr<heap<crypto_segment> > tas;
397 infinint initial_shift;
398 bool reof;
399 trailing_clear_data_callback trailing_clear_data;
400 std::unique_ptr<crypto_segment> ptr;
401 infinint index_num;
402 bool read_ahead_set;
403 infinint read_ahead_offset;
404 libthreadar::mutex ra_cntrl;
405
406
407 // initialized by inherited_run() / get_ready_for_new_offset()
408
409 U_I encrypted_buf_size;
410
411 // fields accessible by both the caller and the read_below thread
412
413 infinint skip_to;
414 tronco_flags flag;
415 infinint clear_flow_start;
416 infinint pos_in_flow;
417
418 void work();
419 infinint get_ready_for_new_offset();
420 void send_flag_to_workers(tronco_flags theflag);
421
422 // same function as the tronconneuse::position_clear2crypt
423 void position_clear2crypt(const infinint & pos,
424 infinint & file_buf_start,
425 infinint & clear_buf_start,
426 infinint & pos_in_buf,
427 infinint & block_num);
428
429 // propagate read_ahead request if necessary
430 void check_read_ahead();
431 };
432
433
435 //
436 // write_below subthread used by parallel_tronconneuse
437 // to gather and write down encrypted data work from workers
438 //
439
440 class write_below: public libthreadar::thread
441 {
442 public:
443 write_below(const std::shared_ptr<libthreadar::ratelier_gather<crypto_segment> > & from_workers,
444 const std::shared_ptr<libthreadar::barrier> & waiter,
445 U_I num_workers,
446 generic_file* encrypted_side,
447 const std::shared_ptr<heap<crypto_segment> > xtas);
448
449 ~write_below() { cancel(); join(); };
450
451 bool exception_pending() const { return error; };
452 const infinint & get_error_block() const { return error_block; };
453
454 protected:
455 virtual void inherited_run() override;
456
457 private:
458 std::shared_ptr<libthreadar::ratelier_gather<crypto_segment> > workers;
459 std::shared_ptr<libthreadar::barrier> waiting;
460 U_I num_w;
461 U_I cur_num_w;
462 generic_file* encrypted;
463 std::shared_ptr<heap<crypto_segment> > tas;
464 bool error;
465 infinint error_block; // last crypto block before error
466 std::deque<std::unique_ptr<crypto_segment> >ones;
467 std::deque<signed int> flags;
468
469 void work();
470 };
471
472
474 //
475 // the crypto_worker threads performing ciphering/deciphering
476 // of many data blocks in parallel
477 //
478
479
480 class crypto_worker: public libthreadar::thread
481 {
482 public:
483 crypto_worker(std::shared_ptr<libthreadar::ratelier_scatter <crypto_segment> > & read_side,
484 std::shared_ptr<libthreadar::ratelier_gather <crypto_segment> > & write_side,
485 std::shared_ptr<libthreadar::barrier> waiter,
486 std::unique_ptr<crypto_module> && ptr,
487 bool encrypt);
488
489 virtual ~crypto_worker() { cancel(); join(); };
490
491 protected:
492 virtual void inherited_run() override;
493
494 private:
495 enum class status { fine, inform, sent };
496
497 std::shared_ptr<libthreadar::ratelier_scatter <crypto_segment> > & reader;
498 std::shared_ptr<libthreadar::ratelier_gather <crypto_segment> > & writer;
499 std::shared_ptr<libthreadar::barrier> waiting;
500 std::unique_ptr<crypto_module> crypto;
501 bool do_encrypt; // if false do decrypt
502 std::unique_ptr<crypto_segment> ptr;
503 unsigned int slot;
504 status abort;
505
506 void work();
507 };
508
509
511
512} // end of namespace
513
514#endif
archive_version.hpp
class archive_version that rules which archive format to follow
libdar::archive_version
class archive_version manages the version of the archive format
Definition archive_version.hpp:47
libdar::generic_file
this is the interface class from which all other data transfer classes inherit
Definition generic_file.hpp:77
libdar::generic_file::is_terminated
bool is_terminated() const
Definition generic_file.hpp:328
libdar::generic_file::sync_write
void sync_write()
write any pending data
libdar::infinint
the arbitrary large positive integer class
Definition real_infinint.hpp:62
libdar::parallel_tronconneuse
this is a partial implementation of the generic_file interface to cypher/decypher data block by block...
Definition parallel_tronconneuse.hpp:89
libdar::parallel_tronconneuse::send_read_order
bool send_read_order(tronco_flags order, const infinint &for_offset=0)
send and order to subthreads and gather acks from them
libdar::parallel_tronconneuse::go_read
void go_read()
wake up threads in read mode when necessary
libdar::parallel_tronconneuse::inherited_terminate
virtual void inherited_terminate() override
this protected inherited method is now private for inherited classes of tronconneuse
libdar::parallel_tronconneuse::inherited_write
virtual void inherited_write(const char *a, U_I size) override
inherited from generic_file
libdar::parallel_tronconneuse::get_clear_block_size
virtual U_32 get_clear_block_size() const override
returns the block size given to constructor
Definition parallel_tronconneuse.hpp:153
libdar::parallel_tronconneuse::run_threads
void run_threads()
reset the interthread datastructure and launch the threads
libdar::parallel_tronconneuse::inherited_sync_write
virtual void inherited_sync_write() override
this protected inherited method is now private for inherited classes of tronconneuse
libdar::parallel_tronconneuse::get_position
virtual infinint get_position() const override
inherited from generic_file
Definition parallel_tronconneuse.hpp:132
libdar::parallel_tronconneuse::join_threads
void join_threads()
wait for threads to finish and eventually rethrow their exceptions in current thread
libdar::parallel_tronconneuse::write_end_of_file
virtual void write_end_of_file() override
in write_only mode indicate that end of file is reached
Definition parallel_tronconneuse.hpp:140
libdar::parallel_tronconneuse::skippable
virtual bool skippable(skippability direction, const infinint &amount) override
inherited from generic_file
libdar::parallel_tronconneuse::purge_unack_stop_order
bool purge_unack_stop_order(const infinint &pos=0)
removing the ignore_stop_acks pending on the pipe
libdar::parallel_tronconneuse::set_callback_trailing_clear_data
virtual void set_callback_trailing_clear_data(trailing_clear_data_callback call_back) override
libdar::parallel_tronconneuse::parallel_tronconneuse
parallel_tronconneuse(parallel_tronconneuse &&ref)=default
move constructor
libdar::parallel_tronconneuse::crypto
std::unique_ptr< crypto_module > crypto
the crypto module use to cipher / uncipher block of data
Definition parallel_tronconneuse.hpp:198
libdar::parallel_tronconneuse::purge_ratelier_from_next_order
tronco_flags purge_ratelier_from_next_order(infinint pos=0)
purge the ratelier from the next order which is provided as returned value
libdar::parallel_tronconneuse::skip_to_eof
virtual bool skip_to_eof() override
inherited from generic_file
libdar::parallel_tronconneuse::reading_ver
archive_version reading_ver
archive format we follow
Definition parallel_tronconneuse.hpp:197
libdar::parallel_tronconneuse::parallel_tronconneuse
parallel_tronconneuse(U_I workers, U_32 block_size, generic_file &encrypted_side, const archive_version &reading_ver, std::unique_ptr< crypto_module > &ptr)
This is the constructor.
libdar::parallel_tronconneuse::skip
virtual bool skip(const infinint &pos) override
inherited from generic_file
libdar::parallel_tronconneuse::~parallel_tronconneuse
~parallel_tronconneuse() noexcept
destructor
libdar::parallel_tronconneuse::operator=
parallel_tronconneuse & operator=(const parallel_tronconneuse &ref)=delete
assignment operator
libdar::parallel_tronconneuse::skip_relative
virtual bool skip_relative(S_I x) override
inherited from generic_file
libdar::parallel_tronconneuse::inherited_truncate
virtual void inherited_truncate(const infinint &pos) override
this prorected inherited method is now private for inherited classed of tronconneuse
Definition parallel_tronconneuse.hpp:174
libdar::parallel_tronconneuse::inherited_read
virtual U_I inherited_read(char *a, U_I size) override
this protected inherited method is now private for inherited classes of tronconneuse
libdar::parallel_tronconneuse::check_bytes_to_skip
bool check_bytes_to_skip
whether to check for bytes to skip
Definition parallel_tronconneuse.hpp:215
libdar::parallel_tronconneuse::send_write_order
void send_write_order(tronco_flags order)
send order in write mode
libdar::parallel_tronconneuse::parallel_tronconneuse
parallel_tronconneuse(const parallel_tronconneuse &ref)=delete
copy constructor
libdar::parallel_tronconneuse::ignore_stop_acks
U_I ignore_stop_acks
how much stop ack still to be read (aborted stop order context)
Definition parallel_tronconneuse.hpp:204
libdar::parallel_tronconneuse::initial_shift
infinint initial_shift
the offset in the "encrypted" below layer at which starts the encrypted data
Definition parallel_tronconneuse.hpp:196
libdar::parallel_tronconneuse::clear_block_size
U_32 clear_block_size
size of a clear block
Definition parallel_tronconneuse.hpp:194
libdar::parallel_tronconneuse::inherited_flush_read
virtual void inherited_flush_read() override
this protected inherited method is now private for inherited classes of tronconneuse
libdar::parallel_tronconneuse::truncatable
virtual bool truncatable(const infinint &pos) const override
inherited from generic_file
Definition parallel_tronconneuse.hpp:130
libdar::parallel_tronconneuse::num_workers
U_I num_workers
number of worker threads
Definition parallel_tronconneuse.hpp:193
libdar::parallel_tronconneuse::find_offset_in_lus_data
bool find_offset_in_lus_data(const infinint &pos)
flush lus_data/lus_flags up to requested pos offset to be found or all data has been removed
libdar::parallel_tronconneuse::stop_threads
void stop_threads()
end threads taking into account the fact they may be suspended on the barrier
libdar::parallel_tronconneuse::t_status
thread_status t_status
wehther child thread are waiting us on the barrier
Definition parallel_tronconneuse.hpp:205
libdar::parallel_tronconneuse::read_refill
void read_refill()
fill lus_data/lus_flags from ratelier_gather if these are empty
libdar::parallel_tronconneuse::set_initial_shift
virtual void set_initial_shift(const infinint &x) override
this method to modify the initial shift. This overrides the constructor "no_initial_shift" of the con...
libdar::parallel_tronconneuse::inherited_read_ahead
virtual void inherited_read_ahead(const infinint &amount) override
this protected inherited method is now private for inherited classes of tronconneuse
libdar::parallel_tronconneuse::current_position
infinint current_position
current position for the upper layer perspective (modified by skip*, inherited_read/write,...
Definition parallel_tronconneuse.hpp:195
libdar::parallel_tronconneuse::join_workers_only
void join_workers_only()
call by join_threads() below just code simplification around exception handling
crypto_module.hpp
per block cryptography implementation
crypto_segment.hpp
defines unit block of information ciphered as once
libdar::trailing_clear_data_callback
infinint(* trailing_clear_data_callback)(generic_file &below, const archive_version &reading_ver)
the trailing_clear_data_callback call back is a mean by which the upper layer cat tell when encrypted...
Definition proto_tronco.hpp:60
libdar::tronco_flags
tronco_flags
status flags used between parallel_tronconneuse and its sub-threads
Definition parallel_tronconneuse.hpp:64
heap.hpp
heap data structure (relying on FIFO)
infinint.hpp
switch module to limitint (32 ou 64 bits integers) or infinint
my_config.h
include macro defined by the configure script and some specific additional ones
libdar
libdar namespace encapsulate all libdar symbols
Definition archive.hpp:47
proto_tronco.hpp
defines common interface for tronconneuse and parallel_tronconneuse